{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "aba02d26-9508-4550-986c-a529fd3dae5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href=\"/pipeline/#/experiments/details/52b34c3f-6851-4467-8c67-9b93106fb162\" target=\"_blank\" >Experiment details</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<a href=\"/pipeline/#/runs/details/39a9b759-b89d-4261-9adc-e6d192862ab9\" target=\"_blank\" >Run details</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import kfp\n",
    "import kfp.components as components\n",
    "\n",
    "# https://www.kubeflow.org/docs/components/pipelines/sdk/pipelines-metrics/\n",
    "#https://elyra.readthedocs.io/en/latest/recipes/visualizing-output-in-the-kfp-ui.html\n",
    "\n",
    "def get_data_batch() -> NamedTuple('Outputs', [('mlpipeline_metrics', 'Metrics')]):\n",
    "    print(\"getting data\")\n",
    "    import json\n",
    "    \n",
    "    accuracy = 0.9\n",
    "    metrics = {\n",
    "        'metrics': [{\n",
    "          'name': 'accuracy-score', # The name of the metric. Visualized as the column name in the runs table.\n",
    "          'numberValue':  accuracy, # The value of the metric. Must be a numeric value.\n",
    "          'format': \"PERCENTAGE\",   # The optional format of the metric. Supported values are \"RAW\" (displayed in raw format) and \"PERCENTAGE\" (displayed in percentage format).\n",
    "        }]\n",
    "    }\n",
    "    return [json.dumps(metrics)]\n",
    "\n",
    "\n",
    "def get_latest_data() -> NamedTuple('VisualizationOutput', [('mlpipeline_ui_metadata', 'UI_metadata')]):\n",
    "    print(\"Getting latest data\")\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    import json\n",
    "    import pandas as pd\n",
    "    \n",
    "    matrix = [\n",
    "    ['yummy', 'yummy', 10],\n",
    "    ['yummy', 'not yummy', 2],\n",
    "    ['not yummy', 'yummy', 6],\n",
    "    ['not yummy', 'not yummy', 7]\n",
    "    ]\n",
    "\n",
    "    df = pd.DataFrame(matrix,columns=['target','predicted','count'])\n",
    "\n",
    "    metadata = {\n",
    "        \"outputs\": [\n",
    "            {\n",
    "                \"type\": \"confusion_matrix\",\n",
    "                \"format\": \"csv\",\n",
    "                \"schema\": [\n",
    "                    {\n",
    "                        \"name\": \"target\",\n",
    "                        \"type\": \"CATEGORY\"\n",
    "                    },\n",
    "                    {\n",
    "                        \"name\": \"predicted\",\n",
    "                        \"type\": \"CATEGORY\"\n",
    "                    },\n",
    "                    {\n",
    "                        \"name\": \"count\",\n",
    "                        \"type\": \"NUMBER\"\n",
    "                    }\n",
    "                ],\n",
    "                \"source\": df.to_csv(header=False, index=False),\n",
    "                \"storage\": \"inline\",\n",
    "                \"labels\": [\n",
    "                    \"yummy\",\n",
    "                    \"not yummy\"\n",
    "                ]\n",
    "            }\n",
    "        ]\n",
    "    }\n",
    "    \n",
    "    from collections import namedtuple\n",
    "    visualization_output = namedtuple('VisualizationOutput', ['mlpipeline_ui_metadata'])\n",
    "    return visualization_output(json.dumps(metadata))\n",
    "    \n",
    "\n",
    "        \n",
    "        \n",
    "from typing import NamedTuple\n",
    "def reshape_data() -> NamedTuple('MyDivmodOutput', [('mlpipeline_ui_metadata', 'UI_metadata'), ('mlpipeline_metrics', 'Metrics')]):\n",
    "    print(\"reshaping data\")\n",
    "    \n",
    "    \n",
    "    # Exports a sample tensorboard:\n",
    "    metadata = {\n",
    "        'outputs': [\n",
    "            {\n",
    "                # Markdown that is hardcoded inline\n",
    "                'storage': 'inline',\n",
    "                'source': '''# Inline Markdown\n",
    "* [Kubeflow official doc](https://www.kubeflow.org/).\n",
    "''',\n",
    "                'type': 'markdown',\n",
    "            },\n",
    "            {\n",
    "                # Markdown that is read from a file\n",
    "                'source': 'https://raw.githubusercontent.com/kubeflow/pipelines/master/README.md',\n",
    "                # Alternatively, use Google Cloud Storage for sample.\n",
    "                # 'source': 'gs://jamxl-kfp-bucket/v2-compatible/markdown/markdown_example.md',\n",
    "                'type': 'markdown',\n",
    "            }]\n",
    "    }\n",
    "\n",
    "    # Exports two sample metrics:\n",
    "    metrics = {\n",
    "      'metrics': [{\n",
    "          'name': 'quotient',\n",
    "          'numberValue':  float(2),\n",
    "        },{\n",
    "          'name': 'remainder',\n",
    "          'numberValue':  float(3),\n",
    "        }]}\n",
    "    \n",
    "    from collections import namedtuple\n",
    "    import json\n",
    "    \n",
    "    divmod_output = namedtuple('MyDivmodOutput', ['mlpipeline_ui_metadata', 'mlpipeline_metrics'])\n",
    "    return divmod_output(json.dumps(metadata), json.dumps(metrics))\n",
    "\n",
    "\n",
    "def model_building(no_epochs:int):\n",
    "    print(\"model building\")\n",
    "    print(no_epochs)\n",
    "    print(type(no_epochs))\n",
    "    \n",
    "    \n",
    "\n",
    "comp_get_data_batch = components.create_component_from_func(get_data_batch,base_image=\"public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-tensorflow-full:v1.4\")\n",
    "comp_get_latest_data = components.create_component_from_func(get_latest_data,base_image=\"public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-tensorflow-full:v1.4\")\n",
    "comp_reshape_data = components.create_component_from_func(reshape_data,base_image=\"public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-tensorflow-full:v1.4\")\n",
    "comp_model_building = components.create_component_from_func(model_building,base_image=\"public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-tensorflow-full:v1.4\")\n",
    "\n",
    "\n",
    "@kfp.dsl.pipeline(\n",
    "   name='output_test',\n",
    "   description='test'\n",
    ")\n",
    "def output_test(no_epochs:int):\n",
    "    step1_1 = comp_get_data_batch()\n",
    "    step1_2 = comp_get_latest_data()\n",
    "    \n",
    "    step2 = comp_reshape_data()\n",
    "    step2.after(step1_1)\n",
    "    step2.after(step1_2)\n",
    "    \n",
    "    step3 = comp_model_building(no_epochs)\n",
    "    step3.after(step2)\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    client = kfp.Client()\n",
    "\n",
    "    arguments = {\n",
    "        \"no_epochs\" : 3\n",
    "    }\n",
    "\n",
    "    run_directly = 1\n",
    "    \n",
    "    if (run_directly == 1):\n",
    "        client.create_run_from_pipeline_func(output_test,arguments=arguments,experiment_name=\"test\")\n",
    "    else:\n",
    "        kfp.compiler.Compiler().compile(pipeline_func=output_test,package_path='output_test.yaml')\n",
    "        client.upload_pipeline_version(pipeline_package_path='output_test.yaml',pipeline_version_name=\"0.4\",pipeline_name=\"pipeline test\",description=\"just for testing\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f75b4dec-c66e-4689-9384-d18521624e70",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ce3bae5-730a-428f-afab-57ce4b67ae99",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "create artifacts, kfpv1\n",
    "\n",
    "def get_data_batch(metadata_data_batch : kfp.components.OutputPath()):\n",
    "    print(\"getting data\")\n",
    "    import json\n",
    "\n",
    "    metadata = {\n",
    "        'outputs' : [\n",
    "        # Markdown that is hardcoded inline\n",
    "        {\n",
    "          'storage': 'inline',\n",
    "          'source': '# Inline Markdown\\n[A link](https://www.kubeflow.org/)',\n",
    "          'type': 'markdown',\n",
    "        }]\n",
    "    }\n",
    "    \n",
    "    with open(metadata_data_batch, 'w') as metadata_file:\n",
    "        json.dump(metadata, metadata_file)\n",
    "\"\"\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
